Disco Diffusion
Contents
Disco Diffusion#
This is an simple way of creating compelling Disco Diffusion artworks for generative artists.
References:#
Note
Install ekorpkit package first.
Set logging level to Warning, if you don’t want to see verbose logging.
If you run this notebook in Colab, set Hardware accelerator to GPU.
!pip install ekorpkit[disco] exit()
from ekorpkit import eKonf
eKonf.setLogger("INFO")
eKonf.set_cuda(device=0)
print("version:", eKonf.__version__)
is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
eKonf.mount_google_drive(
workspace="MyDrive/colab_workspace", project="disco-imagen"
)
print("evironment varialbles:")
eKonf.print(eKonf.env().dict())
INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB']
INFO:ekorpkit.base:Google Colab not detected.
version: 0.1.36+2.g2d58d48.dirty
is notebook? True
is colab? False
evironment varialbles:
{'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',
'CUDA_VISIBLE_DEVICES': '0',
'EKORPKIT_CONFIG_DIR': '/workspace/projects/ekorpkit-book/config',
'EKORPKIT_DATA_DIR': None,
'EKORPKIT_LOG_LEVEL': 'INFO',
'EKORPKIT_PROJECT': 'ekorpkit-book',
'EKORPKIT_WORKSPACE_ROOT': '/workspace',
'KMP_DUPLICATE_LIB_OK': 'TRUE',
'NUM_WORKERS': 230}
cfg = eKonf.compose("model/disco")
disco = eKonf.instantiate(cfg)
INFO:ekorpkit.base:Google Colab not detected.
INFO:ekorpkit.models.dalle.base:> downloading models...
INFO:ekorpkit.models.disco.base:Downloading model model_256 from {'SHA': '983e3de6f95c88c81b2ca7ebb2c217933be1973b1ff058776b970f901584613a', 'filename': '256x256_diffusion_uncond.pt', 'link': 'https://openaipublic.blob.core.windows.net/diffusion/jul-2021/256x256_diffusion_uncond.pt', 'link_fb': 'https://www.dropbox.com/s/9tqnqo930mpnpcn/256x256_diffusion_uncond.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/256x256_diffusion_uncond.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/256x256_diffusion_uncond.pt'}
INFO:ekorpkit.models.disco.utils:Model model_256 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model_512 from {'SHA': '9c111ab89e214862b76e1fa6a1b3f1d329b1a88281885943d2cdbe357ad57648', 'filename': '512x512_diffusion_uncond_finetune_008100.pt', 'link': 'https://the-eye.eu/public/AI/models/512x512_diffusion_unconditional_ImageNet/512x512_diffusion_uncond_finetune_008100.pt', 'link_fb': 'https://huggingface.co/lowlevelware/512x512_diffusion_unconditional_ImageNet/resolve/main/512x512_diffusion_uncond_finetune_008100.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/512x512_diffusion_uncond_finetune_008100.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/512x512_diffusion_uncond_finetune_008100.pt'}
INFO:ekorpkit.models.disco.utils:Model model_512 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model_secondary from {'SHA': '983e3de6f95c88c81b2ca7ebb2c217933be1973b1ff058776b970f901584613a', 'filename': 'secondary_model_imagenet_2.pth', 'link': 'https://huggingface.co/spaces/huggi/secondary_model_imagenet_2.pth/resolve/main/secondary_model_imagenet_2.pth', 'link_fb': 'https://the-eye.eu/public/AI/models/v-diffusion/secondary_model_imagenet_2.pth', 'archive_path': '/workspace/data/archive/disco-imagen/models/secondary_model_imagenet_2.pth', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/secondary_model_imagenet_2.pth'}
INFO:ekorpkit.models.disco.utils:Model model_secondary already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model MiDaS from {'filename': 'dpt_large-midas-2f21e586.pt', 'link': 'https://github.com/intel-isl/DPT/releases/download/1_0/dpt_large-midas-2f21e586.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/dpt_large-midas-2f21e586.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/dpt_large-midas-2f21e586.pt'}
INFO:ekorpkit.models.disco.utils:Model MiDaS already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model-lpips from {'filename': 'vgg16-397923af.pth', 'link': 'https://download.pytorch.org/models/vgg16-397923af.pth', 'archive_path': '/workspace/data/archive/disco-imagen/model-lpips/vgg16-397923af.pth', 'path': '/root/.cache/torch/hub/checkpoints/vgg16-397923af.pth'}
INFO:ekorpkit.models.disco.utils:Model model-lpips already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model AdaBins from {'filename': 'AdaBins_nyu.pt', 'link': 'https://cloudflare-ipfs.com/ipfs/Qmd2mMnDLWePKmgfS8m6ntAg4nhV5VkUyAydYBp8cWWeB7/AdaBins_nyu.pt', 'archive_path': '/workspace/data/archive/disco-imagen/pretrained/AdaBins_nyu.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/pretrained/AdaBins_nyu.pt'}
INFO:ekorpkit.models.disco.utils:Model AdaBins already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50 from {'filename': 'RN50.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50.pt', 'path': '/root/.cache/clip/RN50.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN101 from {'filename': 'RN101.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN101.pt', 'path': '/root/.cache/clip/RN101.pt'}
INFO:ekorpkit.models.disco.utils:Model RN101 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x4 from {'filename': 'RN50x4.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x4.pt', 'path': '/root/.cache/clip/RN50x4.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x4 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x16 from {'filename': 'RN50x16.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x16.pt', 'path': '/root/.cache/clip/RN50x16.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x16 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x64 from {'filename': 'RN50x64.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x64.pt', 'path': '/root/.cache/clip/RN50x64.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x64 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-B-32 from {'filename': 'ViT-B-32.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-B-32.pt', 'path': '/root/.cache/clip/ViT-B-32.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-B-32 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-B-16 from {'filename': 'ViT-B-16.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-B-16.pt', 'path': '/root/.cache/clip/ViT-B-16.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-B-16 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-L-14 from {'filename': 'ViT-L-14.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-L-14.pt', 'path': '/root/.cache/clip/ViT-L-14.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-L-14 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.dalle.base:> loading modules...
INFO:ekorpkit.utils.lib:clip not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/CLIP as clip
INFO:ekorpkit.utils.lib:guided_diffusion.script_util not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/guided-diffusion as guided_diffusion.script_util
INFO:ekorpkit.utils.lib:resize_right not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/ResizeRight as resize_right
INFO:ekorpkit.utils.lib:py3d_tools not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/pytorch3d-lite as py3d_tools
INFO:ekorpkit.utils.lib:midas.dpt_depth not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/MiDaS as midas.dpt_depth
INFO:ekorpkit.utils.lib:utils not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/MiDaS as midas_utils
INFO:ekorpkit.utils.lib:infer not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/AdaBins as infer
INFO:ekorpkit.utils.lib:utils.flow_viz not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/RAFT/core as flow_viz
INFO:ekorpkit.utils.lib:utils.utils not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/RAFT/core as utils.utils
INFO:ekorpkit.utils.lib:raft imported
INFO:ekorpkit.models.dalle.base:> loading models...
INFO:ekorpkit.models.disco.base:> loading diffusion models...
INFO:ekorpkit.models.disco.base:Using device:cuda:0
INFO:ekorpkit.models.disco.base:Disabling CUDNN for A100 gpu
INFO:ekorpkit.models.disco.base:> loading clip models...
Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
Loading model from: /opt/conda/lib/python3.8/site-packages/lpips/weights/v0.1/vgg.pth
text_prompts = "Members of the Federal Reserve Board are convened to cut the target interest rates, surrounding by doves, in a large tower."
text_prompts = 'At a special meeting, hawkish central bankers are poised to raise the target rates, Trending on artstation.'
text_prompts = "Mt. Halla's beautiful flowers, photorealistic"
batch_name = "halla"
disco.imagine(
text_prompts,
batch_name=batch_name,
n_samples=4,
show_collage=True,
width_height = [768, 512],
clip_guidance_scale=5000,
)
INFO:ekorpkit.models.disco.base:Seed used: 362489651
INFO:ekorpkit.models.disco.base: >> elapsed time to diffuse: 0:11:23.813152
INFO:ekorpkit.models.dalle.base:Merging config with args: {}
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 1072306340
INFO:ekorpkit.models.dalle.base:Prompt: Mt. Halla's beautiful flowers, photorealistic
4 samples generated to /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla
text prompts: {0: ["Mt. Halla's beautiful flowers, photorealistic"]}
sample image paths:
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(7)_0000.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(7)_0001.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(7)_0002.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(7)_0003.png
collage generated sample images#
disco.collage(
batch_name=batch_name,
batch_num=4,
ncols=2,
num_images=4,
show_filename=True,
fontcolor="black",
)
INFO:ekorpkit.models.dalle.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(4)_settings.yaml
INFO:ekorpkit.models.dalle.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 221812581
INFO:ekorpkit.models.dalle.base:Prompt: Mt. Halla's beautiful flowers, artstation matte painting
INFO:ekorpkit.io.file:Processing [4] files from ['halla(4)_*.png']
disco.collage(
batch_name=batch_name,
batch_num=1,
ncols=2,
num_images=4,
show_filename=True,
fontcolor="black",
)
INFO:ekorpkit.models.dalle.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(1)_settings.yaml
INFO:ekorpkit.models.dalle.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 1828352028
INFO:ekorpkit.models.dalle.base:Prompt: Mt. Halla's beautiful flowers
INFO:ekorpkit.io.file:Processing [4] files from ['halla(1)_*.png']
make gif of progresses of a specific sample image#
disco.make_gif(
batch_name='halla',
batch_num=1,
sample_num=1,
show=False,
duration=100,
force_remake=True,
)
Saved GIF to /workspace/projects/ekorpkit-book/disco-imagen/outputs/halla/halla(1)_0001.gif
show config#
disco.show_config(batch_name=batch_name, batch_num=4)
INFO:ekorpkit.models.disco.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/halla/halla(4)_settings.yaml
INFO:ekorpkit.models.disco.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Merging config with args: {}
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 3370684779
{'angle': '0:(0)',
'animation_mode': 'None',
'batch_name': 'halla',
'batch_num': 7,
'batch_size': 1,
'calc_frames_skip_steps': 150,
'clamp_grad': True,
'clamp_max': 0.05,
'clip_denoised': False,
'clip_guidance_scale': 5000,
'cut_ic_pow': 1,
'cut_icgray_p': '[0.2]*400+[0]*600',
'cut_innercut': '[4]*400+[12]*600',
'cut_overview': '[12]*400+[4]*600',
'cutn_batches': 4,
'display_rate': 20,
'eta': 0.8,
'extract_nth_frame': 2,
'far_plane': 10000,
'force_flow_generation': False,
'fov': 40,
'frames_scale': 1500,
'frames_skip_steps': '60%',
'fuzzy_prompt': False,
'image_prompts': None,
'init_image': 'None',
'init_scale': 1000,
'intermediate_saves': 250,
'intermediates_in_subfolder': True,
'interp_spline': 'Linear',
'key_frames': True,
'max_frames': 1,
'midas_depth_model': 'dpt_large',
'midas_weight': 0.3,
'n_samples': 4,
'near_plane': 200,
'padding_mode': 'border',
'perlin_init': False,
'perlin_mode': 'mixed',
'persistent_frame_output_in_batch_folder': True,
'rand_mag': 0.05,
'randomize_class': True,
'range_scale': 150,
'resume_from_frame': 'latest',
'resume_run': False,
'retain_overwritten_frames': False,
'rotation_3d_x': '0: (0)',
'rotation_3d_y': '0: (0)',
'rotation_3d_z': '0: (0)',
'run_to_resume': 'latest',
'sampling_mode': 'bicubic',
'sat_scale': 0,
'seed': 3370684779,
'set_seed': 'random_seed',
'show_collage': True,
'side_x': 768,
'side_y': 512,
'skip_augs': False,
'skip_steps': 10,
'start_frame': 0,
'steps': 250,
'steps_per_checkpoint': 2,
'stop_on_next_loop': False,
'text_prompts': {0: ["Mt. Halla's beautiful flowers, artstation matte "
'painting'],
100: ['This set of prompts start at frame 100',
'This prompt has weight five:5']},
'transformation_percent': [0.09],
'translation_x': '0: (0)',
'translation_y': '0: (0)',
'translation_z': '0: (10.0)',
'turbo_mode': False,
'turbo_preroll': 10,
'turbo_steps': '3',
'tv_scale': 0,
'use_horizontal_symmetry': False,
'use_vertical_symmetry': False,
'video_init_blend_mode': 'optical flow',
'video_init_check_consistency': False,
'video_init_clip_guidance_scale': 1000,
'video_init_cutn_batches': 4,
'video_init_file': 'init.mp4',
'video_init_flow_blend': 0.999,
'video_init_flow_warp': True,
'video_init_frames_scale': 15000,
'video_init_frames_skip_steps': '70%',
'video_init_path': '/workspace/projects/ekorpkit-book/disco-imagen/init_images/init.mp4',
'video_init_range_scale': 150,
'video_init_sat_scale': 300,
'video_init_seed_continuity': False,
'video_init_skip_steps': 50,
'video_init_steps': 100,
'video_init_tv_scale': 0.1,
'vr_eye_angle': 0.5,
'vr_ipd': 5.0,
'vr_mode': False,
'width_height': [768, 512],
'zoom': '0: (1), 10: (1.05)'}